package com.panshi.hujin2.httpcrawler.crawler;

import com.panshi.hujin2.httpcrawler.config.BankEmailInfoConfig;
import com.panshi.hujin2.httpcrawler.config.DownloadPathConfig;
import com.panshi.hujin2.httpcrawler.domain.bank.CrawlerBillDoUtil;
import com.panshi.hujin2.httpcrawler.domain.bank.CreditAccountBillDO;
import com.panshi.hujin2.httpcrawler.domain.bank.CreditCardBillDo;
import com.panshi.hujin2.httpcrawler.domain.outlook.OutlookAttachmentDo;
import com.panshi.hujin2.httpcrawler.domain.outlook.OutlookEmailDo;
import com.panshi.hujin2.httpcrawler.domain.outlook.OutlookEmailItemInfo;
import com.panshi.hujin2.httpcrawler.domain.outlook.OutlookInboxDo;
import com.panshi.hujin2.httpcrawler.enumtype.BankTypeEnum;
import com.panshi.hujin2.httpcrawler.pdf.*;
import com.panshi.hujin2.httpcrawler.util.DateUtil;
import com.panshi.hujin2.httpcrawler.util.outlook.*;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.CookieStore;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.cookie.Cookie;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicHeader;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;

import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

/**
 * outlook email crawler
 * @Author: wangxl
 * @Date: 2018/11/20 14:44
 */
@Component
public class OutlookCrawler {

    private static Logger logger = LoggerFactory.getLogger(OutlookCrawler.class);

    /**
     * get email bill from outlook
     * @param inputCookie user email session cookie
     * @param bankType bank type
     * @return
     */
    public static List<CreditAccountBillDO> getBillByOutlook(String inputCookie, int bankType) throws IOException, URISyntaxException {

        CloseableHttpClient httpClient = null;
        try{
            List<CreditAccountBillDO> resList = new ArrayList<CreditAccountBillDO>();

            CookieStore cookieStore = new BasicCookieStore();
            httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build();

            //发起会话数据请求,返回收件箱唯一ID
            OutlookInboxDo inboxDo = getSessionData(httpClient, inputCookie);

            // 解析cookie获取x-owa-canary
            String canary = getCanary(inputCookie, cookieStore);

            // 发请求，获取所有收件箱邮件基本信息
            List<OutlookEmailItemInfo> list = findItems(httpClient,inputCookie,canary,inboxDo.getId(),inboxDo.getTotalCount());

            //获取邮件内容，并下载附件
            for(OutlookEmailItemInfo info:list){
                //TODO 当前版本只支持SANTANDER  ITAU
                // 根据邮件发件人地址和邮件标题关键字 过滤非相关邮件
                if( bankType == BankTypeEnum.SANTANDER.getType() ){
                    if(info.getSenderName().equalsIgnoreCase(BankEmailInfoConfig.staticBankEmailInfoMap.getSantander().get("emailSender"))
                            && info.getSubject().contains(BankEmailInfoConfig.staticBankEmailInfoMap.getSantander().get("titleKeyWord"))){
                    }else{
                        continue;
                    }
                }else if(bankType == BankTypeEnum.BRADESCO.getType()){
                    if(info.getSenderName().equalsIgnoreCase(BankEmailInfoConfig.staticBankEmailInfoMap.getBradesco().get("emailSender"))
                            && info.getSubject().contains(BankEmailInfoConfig.staticBankEmailInfoMap.getBradesco().get("titleKeyWord"))){
                    }else{
                        continue;
                    }
                }else if(bankType == BankTypeEnum.CAIXA.getType()){
                    if(info.getSenderName().equalsIgnoreCase(BankEmailInfoConfig.staticBankEmailInfoMap.getCaixa().get("emailSender"))
                            && info.getSubject().contains(BankEmailInfoConfig.staticBankEmailInfoMap.getCaixa().get("titleKeyWord"))){
                    }else{
                        continue;
                    }
                }else if(bankType == BankTypeEnum.ITAU.getType()){
                    if(info.getSenderName().equalsIgnoreCase(BankEmailInfoConfig.staticBankEmailInfoMap.getItau().get("emailSender"))
                            && info.getSubject().contains(BankEmailInfoConfig.staticBankEmailInfoMap.getItau().get("titleKeyWord"))){
                    }else{
                        continue;
                    }
                }else if(bankType == BankTypeEnum.NUBANK.getType()){
                    if(info.getSenderName().equalsIgnoreCase(BankEmailInfoConfig.staticBankEmailInfoMap.getNubank().get("emailSender"))
                            && info.getSubject().contains(BankEmailInfoConfig.staticBankEmailInfoMap.getNubank().get("titleKeyWord"))){
                    }else{
                        continue;
                    }
                }else{
                    throw new RuntimeException("no supported bank type");
                }
                String emailDateStr = info.getReceiveDate();
                Date emailDate = DateUtil.strToDateLong(emailDateStr);

                OutlookEmailDo email = getOneEmail(httpClient, inputCookie, canary, info.getItemId(), null, null);
                if(email.getAttachmentList().size()>0){
                    for(OutlookAttachmentDo outlookAttachmentDo:email.getAttachmentList()){
                        //如果附件不是pdf文件就跳过
                        if(!outlookAttachmentDo.getAttachmentName().toLowerCase().endsWith(".pdf")){
                            continue;
                        }
                        //开始下载账单PDF
                        String fileName = downloadAttachment(httpClient, inputCookie, inboxDo.getUserEmailAddress(), outlookAttachmentDo, canary, inboxDo.getDownloadToken());
                        //TODO 解析账单,根据银行关键字选择匹配的pdf解析工具

                        CreditCardBillDo creditCardBillDo = BillPdfParseUtil.parsePdf(fileName,bankType);

                        creditCardBillDo.setEmailReceiveDate(emailDate);
                        CreditAccountBillDO creditAccountBillDo = CrawlerBillDoUtil.convert(creditCardBillDo);
                        resList.add(creditAccountBillDo);
                    }
                }
            }
            return resList;

        }finally {
            if(httpClient!=null){
                httpClient.close();
            }
        }
    }
    /**
     * 获取登录会话数据,返回收件箱信息
     *
     * @param cookie
     * @return
     * @throws URISyntaxException
     * @throws ClientProtocolException
     * @throws IOException
     */
    public static OutlookInboxDo getSessionData(CloseableHttpClient httpClient,String cookie)
            throws URISyntaxException, ClientProtocolException, IOException {

        logger.info("outlook爬虫开始获取session会话数据");
        // 1.create request method
        HttpPost post = new HttpPost(new URI("https://outlook.live.com/owa/sessiondata.ashx?app=Mail"));

        // 2.create request header
        Header header = new BasicHeader("Host", "outlook.live.com");
        post.setHeader(header);

        header = new BasicHeader("Connection", "keep-alive");
        post.setHeader(header);

        header = new BasicHeader("Origin", "https://outlook.live.com");
        post.setHeader(header);

        header = new BasicHeader("X-Js-ClientType", "2");
        post.setHeader(header);

        header = new BasicHeader("X-SkipSDCache", "1");
        post.setHeader(header);

        header = new BasicHeader("User-Agent",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36");
        post.setHeader(header);
        header = new BasicHeader("Accept", "*/*");

        post.setHeader(header);
        header = new BasicHeader("Referer", "https://outlook.live.com/");

        post.setHeader(header);
        header = new BasicHeader("Accept-Encoding", "gzip, deflate, br");

        post.setHeader(header);
        header = new BasicHeader("Accept-Language", "zh-CN,zh;q=0.9");//中英文并不影响接收数据
        post.setHeader(header);
        //手机端传入cookie
        header = new BasicHeader("Cookie", cookie);
        post.setHeader(header);

        // 3.send request
        @SuppressWarnings("unused")
        CloseableHttpResponse response = httpClient.execute(post);

        HttpEntity responseEntity = response.getEntity();

        // 5.response content
        String responseContent = EntityUtils.toString(responseEntity, "UTF-8");

        OutlookInboxDo inboxDo = OutlookSessionUtil.getConversationInfo(responseContent);
        if(inboxDo.getId() == null || inboxDo.getId().equals("")){
            throw new RuntimeException();
        }
        logger.info("outlook爬虫开始获取session会话数据成功，返回收件箱唯一ID:"+inboxDo.getId()+"收件箱邮件数量:"+inboxDo.getTotalCount());
        return inboxDo;

    }

    /**
     * 获取收件箱邮件基本信息（邮件标题，邮件唯一ID）
     * @param httpClient
     * @param cookie 外部传入cookie
     * @param canary 参数
     * @param inboxId 收件箱ID
     * @param totalCount 收件箱邮件数量
     * @return
     * @throws URISyntaxException
     * @throws ClientProtocolException
     * @throws IOException
     */
    public static List<OutlookEmailItemInfo> findItems(CloseableHttpClient httpClient,String cookie,String canary,String inboxId,
                                           Integer totalCount) throws URISyntaxException, ClientProtocolException, IOException {

        logger.info("outlook爬虫开始获取收件箱邮件基础信息");
        //urlpostdataTemplate
        String urlpostdataTemplate = "{\"__type\":\"FindItemJsonRequest:#Exchange\",\"Header\":{\"__type\":\"JsonRequestHeaders:#Exchange\",\"RequestServerVersion\":\"V2018_01_08\",\"TimeZoneContext\":{\"__type\":\"TimeZoneContext:#Exchange\",\"TimeZoneDefinition\":{\"__type\":\"TimeZoneDefinitionType:#Exchange\",\"Id\":\"Greenwich Standard Time\"}}},\"Body\":{\"__type\":\"FindItemRequest:#Exchange\",\"ParentFolderIds\":[{\"__type\":\"FolderId:#Exchange\",\"Id\":theReplacePosition}],\"ItemShape\":{\"__type\":\"ItemResponseShape:#Exchange\",\"BaseShape\":\"IdOnly\"},\"ShapeName\":\"MailListItem\",\"Traversal\":\"Shallow\",\"Paging\":{\"__type\":\"IndexedPageView:#Exchange\",\"BasePoint\":\"Beginning\",\"Offset\":0,\"MaxEntriesReturned\":theTotalCount},\"FocusedViewFilter\":0,\"ViewFilter\":\"All\",\"SortOrder\":[{\"__type\":\"SortResults:#Exchange\",\"Order\":\"Descending\",\"Path\":{\"__type\":\"PropertyUri:#Exchange\",\"FieldURI\":\"ReceivedOrRenewTime\"}},{\"__type\":\"SortResults:#Exchange\",\"Order\":\"Descending\",\"Path\":{\"__type\":\"PropertyUri:#Exchange\",\"FieldURI\":\"DateTimeReceived\"}}]}}";

        //replace id
        String urlpostdata =urlpostdataTemplate.replace("theReplacePosition",inboxId);

        urlpostdata =urlpostdata.replace("theTotalCount",totalCount.toString());

        String urlpostdataEncode = UrlUtil.getUrlPostDataEncoderString(urlpostdata);

        // 1.create request method
        HttpPost post = new HttpPost(new URI("https://outlook.live.com/owa/service.svc?action=FindItem&EP=1&app=Mail"));

        // 2.create request header
        Header header = new BasicHeader("Host", "outlook.live.com");
        post.setHeader(header);
        header = new BasicHeader("Connection", "keep-alive");
        post.setHeader(header);
        header = new BasicHeader("Origin", "https://outlook.live.com");
        post.setHeader(header);
        header = new BasicHeader("x-req-source", "Mail");
        post.setHeader(header);
        header = new BasicHeader("x-owa-canary", canary);
        post.setHeader(header);

        header = new BasicHeader("User-Agent",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36");
        post.setHeader(header);

        header = new BasicHeader("x-owa-urlpostdata", urlpostdataEncode);
        post.setHeader(header);

        header = new BasicHeader("action", "FindItem");
        post.setHeader(header);

        header = new BasicHeader("x-owa-correlationid", "");
        post.setHeader(header);

        header = new BasicHeader("ms-cv", "");
        post.setHeader(header);

        header = new BasicHeader("content-type", "application/json; charset=utf-8");
        post.setHeader(header);

        header = new BasicHeader("Accept", "*/*");
        post.setHeader(header);

        header = new BasicHeader("Referer", "https://outlook.live.com/");
        post.setHeader(header);

        header = new BasicHeader("Accept-Encoding", "gzip, deflate, br");
        post.setHeader(header);

        header = new BasicHeader("Accept-Language", "zh-CN,zh;q=0.9");
        // header = new BasicHeader("Accept-Language", "en;q=0.9");//中英文并不影响接收数据
        post.setHeader(header);

        //传入cookie
        header = new BasicHeader("Cookie", cookie);
        post.setHeader(header);

        // 3.send request
        @SuppressWarnings("unused")
        CloseableHttpResponse response = httpClient.execute(post);

        HttpEntity responseEntity = response.getEntity();

        List<OutlookEmailItemInfo> list = OutlookItemIdsParseUtil.getItemIds(EntityUtils.toString(responseEntity, "UTF-8"));

        return list;

    }
    /**
     * 读取单封邮件内容
     *
     */
    public static OutlookEmailDo getOneEmail(CloseableHttpClient httpclient,String cookie,String canary,
                                             String itemId,String correlationid,String mscv) throws ClientProtocolException, IOException, URISyntaxException {

        logger.info("outlook爬虫开始获取收件箱邮件详细信息");

        String urlPostDataContent = "{\"__type\":\"GetItemJsonRequest:#Exchange\",\"Header\":{\"__type\":\"JsonRequestHeaders:#Exchange\",\"RequestServerVersion\":\"V2017_08_18\",\"TimeZoneContext\":{\"__type\":\"TimeZoneContext:#Exchange\",\"TimeZoneDefinition\":{\"__type\":\"TimeZoneDefinitionType:#Exchange\",\"Id\":\"Greenwich Standard Time\"}}},\"Body\":{\"__type\":\"GetItemRequest:#Exchange\",\"ItemShape\":{\"__type\":\"ItemResponseShape:#Exchange\",\"BaseShape\":\"IdOnly\",\"AddBlankTargetToLinks\":true,\"BlockContentFromUnknownSenders\":false,\"BlockExternalImagesIfSenderUntrusted\":true,\"ClientSupportsIrm\":true,\"CssScopeClassName\":\"rps_f624\",\"FilterHtmlContent\":true,\"FilterInlineSafetyTips\":true,\"InlineImageCustomDataTemplate\":\"{id}\",\"InlineImageUrlOnLoadTemplate\":\"InlineImageLoader.GetLoader().Load(this)\",\"InlineImageUrlTemplate\":\"\",\"MaximumBodySize\":2097152,\"MaximumRecipientsToReturn\":20,\"ImageProxyCapability\":\"OwaAndConnectorsProxy\",\"AdditionalProperties\":[{\"__type\":\"PropertyUri:#Exchange\",\"FieldURI\":\"CanDelete\"},{\"__type\":\"ExtendedPropertyUri:#Exchange\",\"PropertySetId\":\"00062008-0000-0000-C000-000000000046\",\"PropertyName\":\"ExplicitMessageCard\",\"PropertyType\":\"String\"}]},\"ItemIds\":[{\"__type\":\"ItemId:#Exchange\",\"Id\":theReplacePosition}],\"ShapeName\":\"ItemNormalizedBody\"}}";

        // replace the email item id
        String urlpostdata = urlPostDataContent.replace("theReplacePosition",itemId);

        //encode
        String urlpostdataEncode = UrlUtil.getUrlPostDataEncoderString(urlpostdata);

        // create request method
        HttpPost post = new HttpPost(new URI("https://outlook.live.com/owa/service.svc?action=GetItem&EP=1&app=Mail"));

        // create request header
        Header header = new BasicHeader("Host", "outlook.live.com");
        post.setHeader(header);
        header = new BasicHeader("Connection", "keep-alive");
        post.setHeader(header);
        header = new BasicHeader("Origin", "https://outlook.live.com");
        post.setHeader(header);
        header = new BasicHeader("x-req-source", "Mail");//可省略
        post.setHeader(header);
        header = new BasicHeader("x-owa-actionsource", "PrefetchFirstN");//可省略
        post.setHeader(header);
        header = new BasicHeader("User-Agent",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36");
        post.setHeader(header);
		header = new BasicHeader("x-owa-urlpostdata",urlpostdataEncode);//可省略
		post.setHeader(header);
        header = new BasicHeader("action", "GetItem");
        post.setHeader(header);
		header = new BasicHeader("x-owa-correlationid", correlationid);//可省略
		post.setHeader(header);
        header = new BasicHeader("ms-cv", mscv);//可省略
        post.setHeader(header);
        header = new BasicHeader("Accept", "*/*");
        post.setHeader(header);
        header = new BasicHeader("Referer", "https://outlook.live.com/");
        post.setHeader(header);
        header = new BasicHeader("Accept-Encoding", "gzip, deflate, br");
        post.setHeader(header);
        header = new BasicHeader("Accept-Language", "zh-CN,zh;q=0.9");
        post.setHeader(header);
        header = new BasicHeader("Cookie",cookie);
        post.setHeader(header);
        header = new BasicHeader("x-owa-canary",canary);
        post.setHeader(header);

        @SuppressWarnings("unused")
        CloseableHttpResponse response = httpclient.execute(post);

        HttpEntity responseEntity = response.getEntity();

        OutlookEmailDo email = OutlookEmailParseUtil.parseOneEmail( EntityUtils.toString(responseEntity, "UTF-8"));

        return email;
    }

    /**
     *  获取canary
     * @param inputCookie
     * @param cookieStore
     * @return
     */
    private static String getCanary(String inputCookie,CookieStore cookieStore){

        String canary = null;
        // 1.从外部传入的cookie提取x-owa-canary
        String[] cookieSplit = inputCookie.split(";");
        for(String t :cookieSplit){
            //去除第一个空格
            t = t.trim();
            if(t.startsWith("X-OWA-CANARY=")){
                String[] canarys = t.split("=");
                canary=canarys[1];
            }
        }
        // 2.从发起第一次请求结果setcookie中提取x-owa-canary
        List<Cookie> cookies = cookieStore.getCookies();
        for(Cookie c:cookies){
            if(c.getName().equalsIgnoreCase("X-OWA-CANARY")){
                canary=c.getValue();
            }
        }
        if(StringUtils.isEmpty(canary)){
            throw new RuntimeException();
        }
        logger.info("success get x-owa-canary="+canary);
        return canary;
    }

    /**
     *
     * @param httpclient
     * @param cookie
     * @param emailAddress
     * @param outlookAttachmentDo
     * @param canary
     * @param downloadToken
     * @return
     * @throws URISyntaxException
     * @throws IOException
     */
    public static String downloadAttachment(CloseableHttpClient httpclient,String cookie,String emailAddress,
                                            OutlookAttachmentDo outlookAttachmentDo,String canary,String downloadToken) throws URISyntaxException, IOException {

        String downloadId = UrlUtil.getURLEncoderString(outlookAttachmentDo.getAttachmentId());
        String uri = "https://attachment.outlook.live.net/owa/"+emailAddress+"/service.svc/s/GetFileAttachment?id="
                +downloadId+"&owa=outlook.live.com&scriptVer=20181112.03&isc=1&X-OWA-CANARY="
                +canary+"&token="+downloadToken+"&isDownload=true&animation=true";

        HttpGet get = new HttpGet(new URI(uri));

        Header header = new BasicHeader("Host", "attachment.outlook.live.net");
        get.setHeader(header);
        header = new BasicHeader("Connection", "keep-alive");
        get.setHeader(header);
        header = new BasicHeader("User-Agent",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36");
        get.setHeader(header);
        header = new BasicHeader("Accept-Encoding", "gzip, deflate, br");
        get.setHeader(header);
        header = new BasicHeader("Accept-Language", "zh-CN,zh;q=0.9");
        get.setHeader(header);
        header = new BasicHeader("Cookie",cookie);
        get.setHeader(header);

        InputStream is = null;
        File file  = null;
        FileOutputStream fos = null;
        BufferedInputStream in= null;
        BufferedOutputStream out= null;
        try {
            CloseableHttpResponse response = httpclient.execute(get);
            is = response.getEntity().getContent();

            // create pdf file
            file = new File(DownloadPathConfig.parentPath+String.valueOf(System.currentTimeMillis())+emailAddress+".pdf");
            fos = new FileOutputStream(file);

            // write to file from response
            in=new BufferedInputStream(is);
            out=new BufferedOutputStream(fos);
            int len=-1;
            byte[] b=new byte[1024];
            while((len=in.read(b))!=-1){
                out.write(b,0,len);
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if(in!=null){ in.close();}
            if(out!=null){ out.close();}
            if(is!=null){ is.close(); }
            if(fos!=null){ fos.close(); }
        }
        // return file absolutepath
        return file.getAbsolutePath();
    }
}
